package edu.hawaii.webspider;

import java.net.URL;
import com.meterware.httpunit.GetMethodWebRequest;
import com.meterware.httpunit.WebConversation;
import com.meterware.httpunit.WebLink;
import com.meterware.httpunit.WebRequest;
import com.meterware.httpunit.WebResponse;

/**
 * A demonstration class illustrating how to retrieve and process web pages using HttpUnit.
 * 
 * @author Philip Johnson
 */
public class WebSpider {

  /** Contains the starting url for the crawl. */
  private final String startUrl;
  
  private int maxPages;

  /**
   * A WebSpider crawls the web and returns info.
   * 
   * @param startUrl The home url for the crawl.
   */
  public WebSpider(String startUrl) {
    this.startUrl = startUrl;
  }
  
  /**
   * Logging interface for WebSpider application.
   */
  WebSpiderLogger logger;
  
  /**
   * Returns the links found at the startUrl.
   * 
   * @return The number of links.
   * @throws Exception if problems occur retrieving the startUrl.
   */
  public WebLink[] getLinks() throws Exception {
    // create the conversation object which will maintain state for us
    WebConversation wc = new WebConversation();
    WebRequest request = new GetMethodWebRequest(this.startUrl);
    WebResponse response = wc.getResponse(request);
    return response.getLinks();
  }

  /**
   * Retrieves the httpunit home page and counts the number of links on it.
   * 
   * @param args Ignored.
   * @throws Exception If problems occur.
   */
  public static void main(String[] args) throws Exception {
    com.meterware.httpunit.HttpUnitOptions.setExceptionsThrownOnScriptError(false);
    com.meterware.httpunit.HttpUnitOptions.setExceptionsThrownOnErrorStatus(false);
    com.meterware.httpunit.HttpUnitOptions.setScriptingEnabled(false); 

    WebSpider spider = new WebSpider(args[1]);
    spider.logger = new WebSpiderLogger();

    Integer maxDepth = Integer.valueOf(args[2]);

    if (args.length > 3 && args[3].equals("-logging")) {
      spider.logger.enable();
    }
    spider.maxPages = maxDepth.intValue();
    spider.processUrl(spider.maxPages);


    if (args[0].equals("-totallinks")) {
      spider.getTotalLinks();
    }
    if (args[0].equals("-mostpopular")) {
      spider.getMostPopular();
    }
  }

  /**
   *  Stores the current URL data for the given site.
   */
  WebAddressDatabase urlList;

  /**
   *  Displays the number of distinct links in urlList.
   */
  public void getTotalLinks() {
    System.out.println("The total number of links discovered while crawling the first " 
        + this.maxPages + " pages accessible from "
        + this.startUrl + " is: " + urlList.getUrlCount());
  }
  /**
   * Displays the most linked-to page in urlList.
   */
  public void getMostPopular() {
    String mostPopularUrl = urlList.getMostPopular();
    System.out.println("The most popular page discovered is " + mostPopularUrl
        + " with " + urlList.getCount(mostPopularUrl) 
        + " pages that point to it.");
  }

  /**
   * Processes a URL, and the pages it links to.  Results are stored
   * in urlList.
   * 
   * @param maxdepth Maximum number of pages to process.  Broken links do not count.
   */
  public void processUrl(int maxdepth) {
    urlList = new WebAddressDatabase();
    urlList.addUrl(startUrl);
    int count = 0;
    String nextUrl;

    while (count < maxdepth && (nextUrl = urlList.getNextUnvisited()) != null) {
      WebSpider urlSearcher = new WebSpider(nextUrl);

      logger.log("Retrieving " + nextUrl);

      try {
        WebLink[] links = urlSearcher.getLinks();

        logger.log("Found " + links.length + " links.");
        
        for (WebLink current : links) {
          URL baseUrl = new URL(nextUrl);
          URL thisUrl;
          String data = "";
          try {
            thisUrl = new URL(baseUrl, current.getURLString());
            data = thisUrl.toExternalForm();

            urlList.addUrl(data);
          }
          catch (Exception e) {
            logger.log("Ignoring link: " + data);
          }
        }
        
        urlList.markUrl(nextUrl);
        count++;
      }
      catch (Exception e) {
        logger.log("Exception thrown: " + e.getClass().getName());
        urlList.markUrl(nextUrl);
      }

    }
  }
}
